package com.edu.chat.web.common.utils;

import org.apache.commons.lang3.StringUtils;

import java.text.Normalizer;
import java.util.regex.Pattern;

/**
 * @program: KQAIA
 * @description:
 * @author: 无恙
 * @create: 2025-04-06 15:51
 **/


public class TextClearUtil {
	// 常见停用词正则（可根据业务扩展）
	private static final String STOP_WORDS = "\\b(the|a|an|and|is|are|in|on|at|to|of|for|with|this|that|these|those)\\b";
	// 特殊字符正则（保留中文标点）
	private static final Pattern SPECIAL_CHARS =
			Pattern.compile("[^\\p{L}\\p{M}\\p{N}\\p{P}\\p{Z}\\p{Sm}\\p{Sc}\\p{Sk}]");
	// URL匹配正则
	private static final Pattern URL_PATTERN =
			Pattern.compile("(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]");
	// 邮箱匹配正则
	private static final Pattern EMAIL_PATTERN =
			Pattern.compile("[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}");

	public static String cleanText(String text) {
		if (StringUtils.isBlank(text)) return "";

		// 1. 统一编码格式
		String cleaned = Normalizer.normalize(text, Normalizer.Form.NFKC);

		// 2. 移除HTML标签
		cleaned = cleaned.replaceAll("<[^>]+>", "");

		// 3. 移除URL和邮箱
		cleaned = URL_PATTERN.matcher(cleaned).replaceAll("");
		cleaned = EMAIL_PATTERN.matcher(cleaned).replaceAll("");

		// 4. 移除特殊字符（保留中文和基础标点）
		cleaned = SPECIAL_CHARS.matcher(cleaned).replaceAll("");

		// 5. 停用词过滤（中英文处理）
		cleaned = cleaned.replaceAll(STOP_WORDS, "")  // 英文停用词
				.replaceAll("\\b(的|是|在|和|了|有|就|都|而|及|与|这|那|你|我|他)\\b", ""); // 中文停用词

		// 6. 统一数字处理（可选策略）
		cleaned = cleaned.replaceAll("\\d+", "<NUM>");

		// 7. 大小写规范化
		cleaned = cleaned.toLowerCase();

		// 8. 去除多余空白
		cleaned = cleaned.replaceAll("\\s+", " ").trim();

		// 9. 处理转义字符
		cleaned = cleaned.replaceAll("[\n\r\t]", " ");

		// 10. 去除零宽字符
		cleaned = cleaned.replaceAll("[\u200B-\u200D\uFEFF]", "");

		return cleaned;
	}


}
